{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Getting started with our ultimate beginner guide!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## This tutorial will walk you through the basics of using the `usb` lighting package. Let's get started by training a FixMatch model on CIFAR-10!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/Zeus/haoc/miniconda/envs/test_semilearn_031/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import semilearn\n",
    "from semilearn import get_dataset, get_data_loader, get_net_builder, get_algorithm, get_config, Trainer\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Step 1: define configs and create config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/bin/sh: 1: netstat: not found\n"
     ]
    }
   ],
   "source": [
    "config = {\n",
    "    'algorithm': 'fixmatch',\n",
    "    'net': 'vit_tiny_patch2_32',\n",
    "    'use_pretrain': True, \n",
    "    'pretrain_path': 'https://github.com/microsoft/Semi-supervised-learning/releases/download/v.0.0.0/vit_tiny_patch2_32_mlp_im_1k_32.pth',\n",
    "\n",
    "    # optimization configs\n",
    "    'epoch': 1,  \n",
    "    'num_train_iter': 5000,  \n",
    "    'num_eval_iter': 500,  \n",
    "    'num_log_iter': 50,  \n",
    "    'optim': 'AdamW',\n",
    "    'lr': 5e-4,\n",
    "    'layer_decay': 0.5,\n",
    "    'batch_size': 16,\n",
    "    'eval_batch_size': 16,\n",
    "\n",
    "\n",
    "    # dataset configs\n",
    "    'dataset': 'cifar10',\n",
    "    'num_labels': 40,\n",
    "    'num_classes': 10,\n",
    "    'img_size': 32,\n",
    "    'crop_ratio': 0.875,\n",
    "    'data_dir': './data',\n",
    "    'ulb_samples_per_class': None,\n",
    "\n",
    "    # algorithm specific configs\n",
    "    'hard_label': True,\n",
    "    'uratio': 2,\n",
    "    'ulb_loss_ratio': 1.0,\n",
    "\n",
    "    # device configs\n",
    "    'gpu': 0,\n",
    "    'world_size': 1,\n",
    "    'distributed': False,\n",
    "    \"num_workers\": 2,\n",
    "}\n",
    "config = get_config(config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Step 2: create model and specify algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "lb count: [4, 4, 4, 4, 4, 4, 4, 4, 4, 4]\n",
      "ulb count: [5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000]\n",
      "Files already downloaded and verified\n",
      "unlabeled data number: 50000, labeled data number 40\n",
      "Create train and test data loaders\n",
      "[!] data loader keys: dict_keys(['train_lb', 'train_ulb', 'eval'])\n",
      "_IncompatibleKeys(missing_keys=['head.weight', 'head.bias'], unexpected_keys=[])\n",
      "Create optimizer and scheduler\n"
     ]
    }
   ],
   "source": [
    "algorithm = get_algorithm(config,  get_net_builder(config.net, from_name=False), tb_log=None, logger=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Step 3: create dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "lb count: [4, 4, 4, 4, 4, 4, 4, 4, 4, 4]\n",
      "ulb count: [5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000, 5000]\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "dataset_dict = get_dataset(config, config.algorithm, config.dataset, config.num_labels, config.num_classes, data_dir=config.data_dir, include_lb_to_ulb=config.include_lb_to_ulb)\n",
    "train_lb_loader = get_data_loader(config, dataset_dict['train_lb'], config.batch_size)\n",
    "train_ulb_loader = get_data_loader(config, dataset_dict['train_ulb'], int(config.batch_size * config.uratio))\n",
    "eval_loader = get_data_loader(config, dataset_dict['eval'], config.eval_batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Step 4: train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0\n",
      "50 iteration USE_EMA: True, train/sup_loss: 1.9210, train/unsup_loss: 0.0000, train/total_loss: 1.9210, train/util_ratio: 0.0000, train/run_time: 0.1366, lr: 0.0005, train/prefecth_time: 0.0040 \n",
      "100 iteration USE_EMA: True, train/sup_loss: 0.9313, train/unsup_loss: 0.0000, train/total_loss: 0.9313, train/util_ratio: 0.0000, train/run_time: 0.1369, lr: 0.0005, train/prefecth_time: 0.0037 \n",
      "150 iteration USE_EMA: True, train/sup_loss: 0.4322, train/unsup_loss: 0.0000, train/total_loss: 0.4322, train/util_ratio: 0.0000, train/run_time: 0.1365, lr: 0.0005, train/prefecth_time: 0.0037 \n",
      "200 iteration USE_EMA: True, train/sup_loss: 0.2618, train/unsup_loss: 0.0000, train/total_loss: 0.2618, train/util_ratio: 0.0000, train/run_time: 0.1383, lr: 0.0005, train/prefecth_time: 0.0040 \n",
      "250 iteration USE_EMA: True, train/sup_loss: 0.4954, train/unsup_loss: 0.0902, train/total_loss: 0.5856, train/util_ratio: 0.0938, train/run_time: 0.1384, lr: 0.0005, train/prefecth_time: 0.0038 \n",
      "300 iteration USE_EMA: True, train/sup_loss: 0.1285, train/unsup_loss: 0.0065, train/total_loss: 0.1350, train/util_ratio: 0.0938, train/run_time: 0.1387, lr: 0.0005, train/prefecth_time: 0.0037 \n",
      "350 iteration USE_EMA: True, train/sup_loss: 0.0994, train/unsup_loss: 0.0367, train/total_loss: 0.1361, train/util_ratio: 0.0938, train/run_time: 0.1388, lr: 0.0005, train/prefecth_time: 0.0040 \n",
      "400 iteration USE_EMA: True, train/sup_loss: 0.0280, train/unsup_loss: 0.0182, train/total_loss: 0.0462, train/util_ratio: 0.1250, train/run_time: 0.1393, lr: 0.0005, train/prefecth_time: 0.0035 \n",
      "450 iteration USE_EMA: True, train/sup_loss: 0.0223, train/unsup_loss: 0.3827, train/total_loss: 0.4050, train/util_ratio: 0.4062, train/run_time: 0.1398, lr: 0.0005, train/prefecth_time: 0.0039 \n",
      "validating...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/Zeus/haoc/miniconda/envs/test_semilearn_031/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix:\n",
      "[[0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 1. 0. 0. 0. 0. 0.]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "500 iteration, USE_EMA: True, train/sup_loss: 0.0236, train/unsup_loss: 0.0640, train/total_loss: 0.0877, train/util_ratio: 0.2812, train/run_time: 0.1395, eval/loss: 5.5914, eval/top-1-acc: 0.1000, eval/balanced_acc: 0.1000, eval/precision: 0.0100, eval/recall: 0.1000, eval/F1: 0.0182, lr: 0.0005, train/prefecth_time: 0.0038 BEST_EVAL_ACC: 0.1000, at 500 iters\n",
      "550 iteration USE_EMA: True, train/sup_loss: 0.0866, train/unsup_loss: 0.2069, train/total_loss: 0.2936, train/util_ratio: 0.2812, train/run_time: 0.1393, lr: 0.0005, train/prefecth_time: 0.0038 \n",
      "600 iteration USE_EMA: True, train/sup_loss: 0.0649, train/unsup_loss: 0.0583, train/total_loss: 0.1232, train/util_ratio: 0.3750, train/run_time: 0.1393, lr: 0.0005, train/prefecth_time: 0.0035 \n",
      "650 iteration USE_EMA: True, train/sup_loss: 0.0256, train/unsup_loss: 0.0846, train/total_loss: 0.1102, train/util_ratio: 0.5625, train/run_time: 0.1402, lr: 0.0005, train/prefecth_time: 0.0031 \n",
      "700 iteration USE_EMA: True, train/sup_loss: 0.0195, train/unsup_loss: 0.2254, train/total_loss: 0.2449, train/util_ratio: 0.5938, train/run_time: 0.1409, lr: 0.0005, train/prefecth_time: 0.0038 \n",
      "750 iteration USE_EMA: True, train/sup_loss: 0.0331, train/unsup_loss: 0.1504, train/total_loss: 0.1835, train/util_ratio: 0.6250, train/run_time: 0.1408, lr: 0.0005, train/prefecth_time: 0.0039 \n",
      "800 iteration USE_EMA: True, train/sup_loss: 0.0006, train/unsup_loss: 0.0241, train/total_loss: 0.0247, train/util_ratio: 0.5938, train/run_time: 0.1414, lr: 0.0005, train/prefecth_time: 0.0037 \n",
      "850 iteration USE_EMA: True, train/sup_loss: 0.0085, train/unsup_loss: 0.1780, train/total_loss: 0.1864, train/util_ratio: 0.5000, train/run_time: 0.1413, lr: 0.0005, train/prefecth_time: 0.0050 \n",
      "900 iteration USE_EMA: True, train/sup_loss: 0.0268, train/unsup_loss: 0.5602, train/total_loss: 0.5870, train/util_ratio: 0.5312, train/run_time: 0.1402, lr: 0.0005, train/prefecth_time: 0.0037 \n",
      "950 iteration USE_EMA: True, train/sup_loss: 0.0060, train/unsup_loss: 0.1938, train/total_loss: 0.1998, train/util_ratio: 0.5312, train/run_time: 0.1400, lr: 0.0005, train/prefecth_time: 0.0035 \n",
      "validating...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/media/Zeus/haoc/miniconda/envs/test_semilearn_031/lib/python3.9/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "confusion matrix:\n",
      "[[0.127 0.    0.    0.    0.484 0.    0.367 0.007 0.004 0.011]\n",
      " [0.    0.277 0.    0.    0.568 0.    0.049 0.039 0.    0.067]\n",
      " [0.    0.    0.    0.    0.93  0.    0.066 0.004 0.    0.   ]\n",
      " [0.    0.    0.    0.002 0.92  0.    0.065 0.012 0.    0.001]\n",
      " [0.    0.    0.    0.    0.995 0.    0.003 0.002 0.    0.   ]\n",
      " [0.    0.    0.    0.    0.955 0.002 0.034 0.009 0.    0.   ]\n",
      " [0.    0.    0.    0.    0.445 0.    0.555 0.    0.    0.   ]\n",
      " [0.    0.    0.    0.    0.847 0.    0.003 0.15  0.    0.   ]\n",
      " [0.002 0.    0.    0.    0.822 0.    0.142 0.009 0.014 0.011]\n",
      " [0.    0.    0.    0.    0.446 0.    0.022 0.005 0.    0.527]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "1000 iteration, USE_EMA: True, train/sup_loss: 0.0011, train/unsup_loss: 0.2126, train/total_loss: 0.2137, train/util_ratio: 0.7500, train/run_time: 0.1401, eval/loss: 2.2794, eval/top-1-acc: 0.2649, eval/balanced_acc: 0.2649, eval/precision: 0.6809, eval/recall: 0.2649, eval/F1: 0.2307, lr: 0.0005, train/prefecth_time: 0.0037 BEST_EVAL_ACC: 0.2649, at 1000 iters\n",
      "1050 iteration USE_EMA: True, train/sup_loss: 0.0556, train/unsup_loss: 0.2425, train/total_loss: 0.2981, train/util_ratio: 0.6562, train/run_time: 0.1413, lr: 0.0005, train/prefecth_time: 0.0038 \n",
      "1100 iteration USE_EMA: True, train/sup_loss: 0.0058, train/unsup_loss: 0.4130, train/total_loss: 0.4188, train/util_ratio: 0.7188, train/run_time: 0.1409, lr: 0.0005, train/prefecth_time: 0.0038 \n",
      "1150 iteration USE_EMA: True, train/sup_loss: 0.0017, train/unsup_loss: 0.4102, train/total_loss: 0.4120, train/util_ratio: 0.6562, train/run_time: 0.1404, lr: 0.0005, train/prefecth_time: 0.0032 \n",
      "1200 iteration USE_EMA: True, train/sup_loss: 0.0031, train/unsup_loss: 0.2593, train/total_loss: 0.2624, train/util_ratio: 0.6562, train/run_time: 0.1407, lr: 0.0005, train/prefecth_time: 0.0035 \n",
      "1250 iteration USE_EMA: True, train/sup_loss: 0.0039, train/unsup_loss: 0.2743, train/total_loss: 0.2783, train/util_ratio: 0.7188, train/run_time: 0.1409, lr: 0.0005, train/prefecth_time: 0.0037 \n",
      "1300 iteration USE_EMA: True, train/sup_loss: 0.0655, train/unsup_loss: 0.1830, train/total_loss: 0.2484, train/util_ratio: 0.6562, train/run_time: 0.1407, lr: 0.0005, train/prefecth_time: 0.0038 \n",
      "1350 iteration USE_EMA: True, train/sup_loss: 0.0027, train/unsup_loss: 0.0781, train/total_loss: 0.0808, train/util_ratio: 0.5625, train/run_time: 0.1414, lr: 0.0005, train/prefecth_time: 0.0035 \n",
      "1400 iteration USE_EMA: True, train/sup_loss: 0.0141, train/unsup_loss: 0.2426, train/total_loss: 0.2568, train/util_ratio: 0.7188, train/run_time: 0.1403, lr: 0.0005, train/prefecth_time: 0.0036 \n",
      "1450 iteration USE_EMA: True, train/sup_loss: 0.0057, train/unsup_loss: 0.3173, train/total_loss: 0.3229, train/util_ratio: 0.8750, train/run_time: 0.1414, lr: 0.0005, train/prefecth_time: 0.0037 \n",
      "validating...\n",
      "confusion matrix:\n",
      "[[0.922 0.001 0.001 0.001 0.005 0.    0.03  0.001 0.017 0.022]\n",
      " [0.001 0.963 0.    0.    0.003 0.    0.001 0.    0.001 0.031]\n",
      " [0.024 0.    0.436 0.002 0.419 0.005 0.1   0.014 0.    0.   ]\n",
      " [0.01  0.001 0.002 0.5   0.283 0.085 0.085 0.029 0.002 0.003]\n",
      " [0.    0.    0.    0.001 0.981 0.    0.007 0.011 0.    0.   ]\n",
      " [0.001 0.    0.    0.015 0.214 0.692 0.024 0.054 0.    0.   ]\n",
      " [0.001 0.    0.001 0.    0.01  0.001 0.986 0.    0.001 0.   ]\n",
      " [0.005 0.    0.    0.006 0.374 0.008 0.002 0.605 0.    0.   ]\n",
      " [0.023 0.011 0.    0.    0.048 0.    0.025 0.002 0.865 0.026]\n",
      " [0.003 0.012 0.    0.    0.    0.    0.002 0.    0.001 0.982]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "1500 iteration, USE_EMA: True, train/sup_loss: 0.0110, train/unsup_loss: 0.1831, train/total_loss: 0.1942, train/util_ratio: 0.6250, train/run_time: 0.1408, eval/loss: 0.6494, eval/top-1-acc: 0.7932, eval/balanced_acc: 0.7932, eval/precision: 0.8668, eval/recall: 0.7932, eval/F1: 0.7963, lr: 0.0005, train/prefecth_time: 0.0036 BEST_EVAL_ACC: 0.7932, at 1500 iters\n",
      "1550 iteration USE_EMA: True, train/sup_loss: 0.0835, train/unsup_loss: 0.3773, train/total_loss: 0.4608, train/util_ratio: 0.6875, train/run_time: 0.1409, lr: 0.0005, train/prefecth_time: 0.0039 \n",
      "1600 iteration USE_EMA: True, train/sup_loss: 0.0115, train/unsup_loss: 0.5431, train/total_loss: 0.5546, train/util_ratio: 0.7812, train/run_time: 0.1409, lr: 0.0005, train/prefecth_time: 0.0040 \n",
      "1650 iteration USE_EMA: True, train/sup_loss: 0.0255, train/unsup_loss: 0.1681, train/total_loss: 0.1937, train/util_ratio: 0.7188, train/run_time: 0.1412, lr: 0.0004, train/prefecth_time: 0.0039 \n",
      "1700 iteration USE_EMA: True, train/sup_loss: 0.0012, train/unsup_loss: 0.1483, train/total_loss: 0.1495, train/util_ratio: 0.5625, train/run_time: 0.1412, lr: 0.0004, train/prefecth_time: 0.0039 \n",
      "1750 iteration USE_EMA: True, train/sup_loss: 0.0016, train/unsup_loss: 0.0522, train/total_loss: 0.0538, train/util_ratio: 0.6250, train/run_time: 0.1405, lr: 0.0004, train/prefecth_time: 0.0041 \n",
      "1800 iteration USE_EMA: True, train/sup_loss: 0.0082, train/unsup_loss: 0.2608, train/total_loss: 0.2690, train/util_ratio: 0.8125, train/run_time: 0.1415, lr: 0.0004, train/prefecth_time: 0.0037 \n",
      "1850 iteration USE_EMA: True, train/sup_loss: 0.0011, train/unsup_loss: 0.1171, train/total_loss: 0.1182, train/util_ratio: 0.8438, train/run_time: 0.1412, lr: 0.0004, train/prefecth_time: 0.0042 \n",
      "1900 iteration USE_EMA: True, train/sup_loss: 0.0348, train/unsup_loss: 0.1308, train/total_loss: 0.1656, train/util_ratio: 0.6562, train/run_time: 0.1411, lr: 0.0004, train/prefecth_time: 0.0037 \n",
      "1950 iteration USE_EMA: True, train/sup_loss: 0.0104, train/unsup_loss: 0.0852, train/total_loss: 0.0956, train/util_ratio: 0.5625, train/run_time: 0.1412, lr: 0.0004, train/prefecth_time: 0.0040 \n",
      "validating...\n",
      "confusion matrix:\n",
      "[[0.946 0.003 0.001 0.001 0.002 0.    0.007 0.    0.019 0.021]\n",
      " [0.    0.979 0.    0.    0.    0.    0.001 0.    0.001 0.019]\n",
      " [0.04  0.    0.701 0.006 0.177 0.012 0.048 0.016 0.    0.   ]\n",
      " [0.009 0.001 0.004 0.769 0.059 0.1   0.039 0.015 0.002 0.002]\n",
      " [0.    0.    0.001 0.005 0.96  0.001 0.008 0.024 0.001 0.   ]\n",
      " [0.001 0.    0.001 0.024 0.015 0.915 0.007 0.037 0.    0.   ]\n",
      " [0.001 0.    0.003 0.    0.002 0.001 0.992 0.    0.001 0.   ]\n",
      " [0.005 0.    0.003 0.009 0.109 0.03  0.001 0.843 0.    0.   ]\n",
      " [0.03  0.007 0.    0.    0.005 0.    0.005 0.002 0.931 0.02 ]\n",
      " [0.004 0.025 0.    0.    0.    0.    0.    0.    0.001 0.97 ]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "2000 iteration, USE_EMA: True, train/sup_loss: 0.0046, train/unsup_loss: 0.1097, train/total_loss: 0.1143, train/util_ratio: 0.7500, train/run_time: 0.1419, eval/loss: 0.3166, eval/top-1-acc: 0.9006, eval/balanced_acc: 0.9006, eval/precision: 0.9099, eval/recall: 0.9006, eval/F1: 0.8998, lr: 0.0004, train/prefecth_time: 0.0040 BEST_EVAL_ACC: 0.9006, at 2000 iters\n",
      "2050 iteration USE_EMA: True, train/sup_loss: 0.0362, train/unsup_loss: 0.0596, train/total_loss: 0.0959, train/util_ratio: 0.7812, train/run_time: 0.1412, lr: 0.0004, train/prefecth_time: 0.0040 \n",
      "2100 iteration USE_EMA: True, train/sup_loss: 0.0052, train/unsup_loss: 0.2378, train/total_loss: 0.2430, train/util_ratio: 0.7500, train/run_time: 0.1412, lr: 0.0004, train/prefecth_time: 0.0039 \n",
      "2150 iteration USE_EMA: True, train/sup_loss: 0.0005, train/unsup_loss: 0.1701, train/total_loss: 0.1707, train/util_ratio: 0.8125, train/run_time: 0.1414, lr: 0.0004, train/prefecth_time: 0.0036 \n",
      "2200 iteration USE_EMA: True, train/sup_loss: 0.0006, train/unsup_loss: 0.0899, train/total_loss: 0.0905, train/util_ratio: 0.7500, train/run_time: 0.1408, lr: 0.0004, train/prefecth_time: 0.0036 \n",
      "2250 iteration USE_EMA: True, train/sup_loss: 0.0319, train/unsup_loss: 0.1628, train/total_loss: 0.1947, train/util_ratio: 0.6562, train/run_time: 0.1414, lr: 0.0004, train/prefecth_time: 0.0034 \n",
      "2300 iteration USE_EMA: True, train/sup_loss: 0.0542, train/unsup_loss: 0.0963, train/total_loss: 0.1504, train/util_ratio: 0.8125, train/run_time: 0.1405, lr: 0.0004, train/prefecth_time: 0.0035 \n",
      "2350 iteration USE_EMA: True, train/sup_loss: 0.0023, train/unsup_loss: 0.0910, train/total_loss: 0.0934, train/util_ratio: 0.7812, train/run_time: 0.1412, lr: 0.0004, train/prefecth_time: 0.0038 \n",
      "2400 iteration USE_EMA: True, train/sup_loss: 0.0260, train/unsup_loss: 0.0474, train/total_loss: 0.0733, train/util_ratio: 0.7188, train/run_time: 0.1413, lr: 0.0004, train/prefecth_time: 0.0037 \n",
      "2450 iteration USE_EMA: True, train/sup_loss: 0.0037, train/unsup_loss: 0.1110, train/total_loss: 0.1147, train/util_ratio: 0.8125, train/run_time: 0.1419, lr: 0.0004, train/prefecth_time: 0.0035 \n",
      "validating...\n",
      "confusion matrix:\n",
      "[[0.954 0.003 0.001 0.001 0.    0.    0.002 0.    0.021 0.018]\n",
      " [0.    0.99  0.    0.    0.    0.    0.    0.    0.    0.01 ]\n",
      " [0.042 0.    0.749 0.006 0.144 0.012 0.034 0.012 0.    0.001]\n",
      " [0.006 0.001 0.001 0.826 0.03  0.096 0.027 0.007 0.003 0.003]\n",
      " [0.    0.    0.001 0.006 0.954 0.003 0.006 0.029 0.001 0.   ]\n",
      " [0.    0.    0.001 0.021 0.009 0.947 0.004 0.018 0.    0.   ]\n",
      " [0.001 0.    0.003 0.    0.001 0.002 0.992 0.    0.001 0.   ]\n",
      " [0.007 0.    0.002 0.013 0.029 0.042 0.    0.907 0.    0.   ]\n",
      " [0.024 0.009 0.    0.    0.001 0.    0.001 0.    0.95  0.015]\n",
      " [0.004 0.025 0.    0.    0.    0.    0.    0.    0.001 0.97 ]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "2500 iteration, USE_EMA: True, train/sup_loss: 0.0010, train/unsup_loss: 0.1853, train/total_loss: 0.1862, train/util_ratio: 0.8750, train/run_time: 0.1411, eval/loss: 0.2508, eval/top-1-acc: 0.9239, eval/balanced_acc: 0.9239, eval/precision: 0.9281, eval/recall: 0.9239, eval/F1: 0.9230, lr: 0.0004, train/prefecth_time: 0.0039 BEST_EVAL_ACC: 0.9239, at 2500 iters\n",
      "2550 iteration USE_EMA: True, train/sup_loss: 0.0608, train/unsup_loss: 0.1851, train/total_loss: 0.2459, train/util_ratio: 0.7188, train/run_time: 0.1410, lr: 0.0004, train/prefecth_time: 0.0036 \n",
      "2600 iteration USE_EMA: True, train/sup_loss: 0.0012, train/unsup_loss: 0.1002, train/total_loss: 0.1014, train/util_ratio: 0.6875, train/run_time: 0.1412, lr: 0.0004, train/prefecth_time: 0.0039 \n",
      "2650 iteration USE_EMA: True, train/sup_loss: 0.0130, train/unsup_loss: 0.0630, train/total_loss: 0.0761, train/util_ratio: 0.7500, train/run_time: 0.1410, lr: 0.0004, train/prefecth_time: 0.0035 \n",
      "2700 iteration USE_EMA: True, train/sup_loss: 0.0062, train/unsup_loss: 0.1358, train/total_loss: 0.1421, train/util_ratio: 0.8750, train/run_time: 0.1408, lr: 0.0004, train/prefecth_time: 0.0039 \n",
      "2750 iteration USE_EMA: True, train/sup_loss: 0.0571, train/unsup_loss: 0.1289, train/total_loss: 0.1860, train/util_ratio: 0.6875, train/run_time: 0.1406, lr: 0.0004, train/prefecth_time: 0.0035 \n",
      "2800 iteration USE_EMA: True, train/sup_loss: 0.0753, train/unsup_loss: 0.1043, train/total_loss: 0.1796, train/util_ratio: 0.7812, train/run_time: 0.1414, lr: 0.0004, train/prefecth_time: 0.0039 \n",
      "2850 iteration USE_EMA: True, train/sup_loss: 0.0027, train/unsup_loss: 0.0043, train/total_loss: 0.0070, train/util_ratio: 0.7188, train/run_time: 0.1410, lr: 0.0004, train/prefecth_time: 0.0037 \n",
      "2900 iteration USE_EMA: True, train/sup_loss: 0.0065, train/unsup_loss: 0.1434, train/total_loss: 0.1499, train/util_ratio: 0.8438, train/run_time: 0.1423, lr: 0.0003, train/prefecth_time: 0.0036 \n",
      "2950 iteration USE_EMA: True, train/sup_loss: 0.0813, train/unsup_loss: 0.3382, train/total_loss: 0.4195, train/util_ratio: 0.7500, train/run_time: 0.1412, lr: 0.0003, train/prefecth_time: 0.0037 \n",
      "validating...\n",
      "confusion matrix:\n",
      "[[0.958 0.002 0.001 0.001 0.    0.    0.002 0.    0.022 0.014]\n",
      " [0.    0.989 0.    0.    0.    0.    0.    0.    0.001 0.01 ]\n",
      " [0.042 0.    0.764 0.007 0.14  0.012 0.024 0.01  0.    0.001]\n",
      " [0.005 0.001 0.003 0.841 0.019 0.095 0.023 0.007 0.003 0.003]\n",
      " [0.    0.    0.002 0.009 0.941 0.006 0.007 0.034 0.001 0.   ]\n",
      " [0.    0.    0.001 0.018 0.007 0.956 0.002 0.016 0.    0.   ]\n",
      " [0.001 0.    0.003 0.    0.    0.002 0.993 0.    0.001 0.   ]\n",
      " [0.005 0.    0.002 0.01  0.012 0.047 0.    0.924 0.    0.   ]\n",
      " [0.016 0.009 0.    0.    0.    0.    0.001 0.    0.964 0.01 ]\n",
      " [0.004 0.027 0.    0.    0.    0.    0.    0.    0.001 0.968]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "3000 iteration, USE_EMA: True, train/sup_loss: 0.0019, train/unsup_loss: 0.0714, train/total_loss: 0.0733, train/util_ratio: 0.7188, train/run_time: 0.1415, eval/loss: 0.2337, eval/top-1-acc: 0.9298, eval/balanced_acc: 0.9298, eval/precision: 0.9330, eval/recall: 0.9298, eval/F1: 0.9290, lr: 0.0003, train/prefecth_time: 0.0036 BEST_EVAL_ACC: 0.9298, at 3000 iters\n",
      "3050 iteration USE_EMA: True, train/sup_loss: 0.0003, train/unsup_loss: 0.2654, train/total_loss: 0.2657, train/util_ratio: 0.6562, train/run_time: 0.1415, lr: 0.0003, train/prefecth_time: 0.0039 \n",
      "3100 iteration USE_EMA: True, train/sup_loss: 0.0526, train/unsup_loss: 0.0743, train/total_loss: 0.1269, train/util_ratio: 0.7812, train/run_time: 0.1411, lr: 0.0003, train/prefecth_time: 0.0035 \n",
      "3150 iteration USE_EMA: True, train/sup_loss: 0.0004, train/unsup_loss: 0.1189, train/total_loss: 0.1193, train/util_ratio: 0.8750, train/run_time: 0.1406, lr: 0.0003, train/prefecth_time: 0.0038 \n",
      "3200 iteration USE_EMA: True, train/sup_loss: 0.0074, train/unsup_loss: 0.1950, train/total_loss: 0.2024, train/util_ratio: 0.6562, train/run_time: 0.1411, lr: 0.0003, train/prefecth_time: 0.0036 \n",
      "3250 iteration USE_EMA: True, train/sup_loss: 0.0459, train/unsup_loss: 0.3374, train/total_loss: 0.3833, train/util_ratio: 0.8438, train/run_time: 0.1411, lr: 0.0003, train/prefecth_time: 0.0037 \n",
      "3300 iteration USE_EMA: True, train/sup_loss: 0.0059, train/unsup_loss: 0.2219, train/total_loss: 0.2278, train/util_ratio: 0.8438, train/run_time: 0.1417, lr: 0.0003, train/prefecth_time: 0.0037 \n",
      "3350 iteration USE_EMA: True, train/sup_loss: 0.0015, train/unsup_loss: 0.0706, train/total_loss: 0.0720, train/util_ratio: 0.7812, train/run_time: 0.1413, lr: 0.0003, train/prefecth_time: 0.0036 \n",
      "3400 iteration USE_EMA: True, train/sup_loss: 0.2013, train/unsup_loss: 0.0645, train/total_loss: 0.2658, train/util_ratio: 0.7812, train/run_time: 0.1415, lr: 0.0003, train/prefecth_time: 0.0038 \n",
      "3450 iteration USE_EMA: True, train/sup_loss: 0.0029, train/unsup_loss: 0.3704, train/total_loss: 0.3733, train/util_ratio: 0.9062, train/run_time: 0.1414, lr: 0.0003, train/prefecth_time: 0.0037 \n",
      "validating...\n",
      "confusion matrix:\n",
      "[[0.957 0.002 0.002 0.001 0.    0.    0.001 0.    0.022 0.015]\n",
      " [0.    0.988 0.    0.    0.    0.    0.    0.    0.001 0.011]\n",
      " [0.043 0.    0.774 0.007 0.134 0.011 0.021 0.009 0.    0.001]\n",
      " [0.003 0.001 0.003 0.862 0.016 0.083 0.023 0.004 0.003 0.002]\n",
      " [0.    0.    0.003 0.01  0.936 0.007 0.008 0.035 0.001 0.   ]\n",
      " [0.    0.    0.001 0.022 0.006 0.955 0.002 0.014 0.    0.   ]\n",
      " [0.001 0.    0.004 0.001 0.    0.002 0.991 0.    0.001 0.   ]\n",
      " [0.005 0.    0.002 0.009 0.006 0.046 0.    0.932 0.    0.   ]\n",
      " [0.014 0.007 0.    0.    0.    0.    0.001 0.    0.969 0.009]\n",
      " [0.004 0.028 0.    0.    0.    0.    0.    0.    0.002 0.966]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "3500 iteration, USE_EMA: True, train/sup_loss: 0.0351, train/unsup_loss: 0.2334, train/total_loss: 0.2685, train/util_ratio: 0.8750, train/run_time: 0.1405, eval/loss: 0.2236, eval/top-1-acc: 0.9330, eval/balanced_acc: 0.9330, eval/precision: 0.9355, eval/recall: 0.9330, eval/F1: 0.9323, lr: 0.0003, train/prefecth_time: 0.0037 BEST_EVAL_ACC: 0.9330, at 3500 iters\n",
      "3550 iteration USE_EMA: True, train/sup_loss: 0.0035, train/unsup_loss: 0.1521, train/total_loss: 0.1556, train/util_ratio: 0.7500, train/run_time: 0.1413, lr: 0.0003, train/prefecth_time: 0.0038 \n",
      "3600 iteration USE_EMA: True, train/sup_loss: 0.0002, train/unsup_loss: 0.3567, train/total_loss: 0.3570, train/util_ratio: 0.8125, train/run_time: 0.1416, lr: 0.0003, train/prefecth_time: 0.0040 \n",
      "3650 iteration USE_EMA: True, train/sup_loss: 0.0004, train/unsup_loss: 0.1816, train/total_loss: 0.1820, train/util_ratio: 0.6875, train/run_time: 0.1410, lr: 0.0003, train/prefecth_time: 0.0039 \n",
      "3700 iteration USE_EMA: True, train/sup_loss: 0.0007, train/unsup_loss: 0.3171, train/total_loss: 0.3178, train/util_ratio: 0.7500, train/run_time: 0.1409, lr: 0.0003, train/prefecth_time: 0.0039 \n",
      "3750 iteration USE_EMA: True, train/sup_loss: 0.0019, train/unsup_loss: 0.2061, train/total_loss: 0.2079, train/util_ratio: 0.8750, train/run_time: 0.1410, lr: 0.0003, train/prefecth_time: 0.0038 \n",
      "3800 iteration USE_EMA: True, train/sup_loss: 0.0040, train/unsup_loss: 0.0473, train/total_loss: 0.0514, train/util_ratio: 0.8750, train/run_time: 0.1416, lr: 0.0003, train/prefecth_time: 0.0037 \n",
      "3850 iteration USE_EMA: True, train/sup_loss: 0.0848, train/unsup_loss: 0.0998, train/total_loss: 0.1846, train/util_ratio: 0.9375, train/run_time: 0.1414, lr: 0.0002, train/prefecth_time: 0.0036 \n",
      "3900 iteration USE_EMA: True, train/sup_loss: 0.0017, train/unsup_loss: 0.0173, train/total_loss: 0.0190, train/util_ratio: 0.7812, train/run_time: 0.1416, lr: 0.0002, train/prefecth_time: 0.0036 \n",
      "3950 iteration USE_EMA: True, train/sup_loss: 0.0003, train/unsup_loss: 0.0839, train/total_loss: 0.0842, train/util_ratio: 0.6875, train/run_time: 0.1406, lr: 0.0002, train/prefecth_time: 0.0038 \n",
      "validating...\n",
      "confusion matrix:\n",
      "[[0.957 0.002 0.002 0.001 0.    0.    0.001 0.    0.022 0.015]\n",
      " [0.    0.988 0.    0.    0.    0.    0.    0.    0.001 0.011]\n",
      " [0.041 0.    0.782 0.006 0.132 0.01  0.019 0.009 0.    0.001]\n",
      " [0.003 0.001 0.003 0.861 0.019 0.084 0.022 0.002 0.003 0.002]\n",
      " [0.    0.    0.003 0.008 0.938 0.008 0.007 0.035 0.001 0.   ]\n",
      " [0.    0.    0.001 0.021 0.006 0.956 0.002 0.014 0.    0.   ]\n",
      " [0.002 0.    0.003 0.001 0.    0.002 0.991 0.    0.001 0.   ]\n",
      " [0.004 0.    0.002 0.008 0.006 0.039 0.    0.94  0.001 0.   ]\n",
      " [0.011 0.006 0.    0.    0.    0.    0.    0.    0.972 0.011]\n",
      " [0.003 0.03  0.    0.    0.    0.    0.    0.    0.002 0.965]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "4000 iteration, USE_EMA: True, train/sup_loss: 0.0075, train/unsup_loss: 0.1623, train/total_loss: 0.1698, train/util_ratio: 0.9062, train/run_time: 0.1419, eval/loss: 0.2220, eval/top-1-acc: 0.9350, eval/balanced_acc: 0.9350, eval/precision: 0.9374, eval/recall: 0.9350, eval/F1: 0.9344, lr: 0.0002, train/prefecth_time: 0.0038 BEST_EVAL_ACC: 0.9350, at 4000 iters\n",
      "4050 iteration USE_EMA: True, train/sup_loss: 0.0002, train/unsup_loss: 0.1693, train/total_loss: 0.1695, train/util_ratio: 0.6250, train/run_time: 0.1410, lr: 0.0002, train/prefecth_time: 0.0031 \n",
      "4100 iteration USE_EMA: True, train/sup_loss: 0.0013, train/unsup_loss: 0.0134, train/total_loss: 0.0147, train/util_ratio: 0.6875, train/run_time: 0.1410, lr: 0.0002, train/prefecth_time: 0.0039 \n",
      "4150 iteration USE_EMA: True, train/sup_loss: 0.0192, train/unsup_loss: 0.0331, train/total_loss: 0.0522, train/util_ratio: 0.5938, train/run_time: 0.1412, lr: 0.0002, train/prefecth_time: 0.0039 \n",
      "4200 iteration USE_EMA: True, train/sup_loss: 0.0165, train/unsup_loss: 0.1861, train/total_loss: 0.2026, train/util_ratio: 0.7812, train/run_time: 0.1409, lr: 0.0002, train/prefecth_time: 0.0038 \n",
      "4250 iteration USE_EMA: True, train/sup_loss: 0.0276, train/unsup_loss: 0.1913, train/total_loss: 0.2189, train/util_ratio: 0.8750, train/run_time: 0.1412, lr: 0.0002, train/prefecth_time: 0.0036 \n",
      "4300 iteration USE_EMA: True, train/sup_loss: 0.0003, train/unsup_loss: 0.1182, train/total_loss: 0.1186, train/util_ratio: 0.6562, train/run_time: 0.1407, lr: 0.0002, train/prefecth_time: 0.0036 \n",
      "4350 iteration USE_EMA: True, train/sup_loss: 0.0138, train/unsup_loss: 0.2013, train/total_loss: 0.2152, train/util_ratio: 0.7188, train/run_time: 0.1411, lr: 0.0002, train/prefecth_time: 0.0039 \n",
      "4400 iteration USE_EMA: True, train/sup_loss: 0.0021, train/unsup_loss: 0.0658, train/total_loss: 0.0679, train/util_ratio: 0.8438, train/run_time: 0.1405, lr: 0.0002, train/prefecth_time: 0.0036 \n",
      "4450 iteration USE_EMA: True, train/sup_loss: 0.0002, train/unsup_loss: 0.1821, train/total_loss: 0.1823, train/util_ratio: 0.6562, train/run_time: 0.1410, lr: 0.0002, train/prefecth_time: 0.0037 \n",
      "validating...\n",
      "confusion matrix:\n",
      "[[0.957 0.002 0.002 0.001 0.    0.    0.    0.    0.022 0.016]\n",
      " [0.    0.989 0.    0.    0.    0.    0.    0.    0.001 0.01 ]\n",
      " [0.042 0.    0.782 0.006 0.134 0.01  0.019 0.006 0.    0.001]\n",
      " [0.003 0.001 0.003 0.861 0.018 0.086 0.022 0.001 0.003 0.002]\n",
      " [0.    0.    0.003 0.007 0.943 0.008 0.007 0.031 0.001 0.   ]\n",
      " [0.    0.    0.001 0.019 0.006 0.96  0.002 0.012 0.    0.   ]\n",
      " [0.002 0.    0.003 0.001 0.    0.002 0.991 0.    0.001 0.   ]\n",
      " [0.004 0.    0.002 0.007 0.006 0.039 0.    0.941 0.001 0.   ]\n",
      " [0.008 0.007 0.    0.    0.    0.    0.    0.    0.975 0.01 ]\n",
      " [0.003 0.031 0.    0.    0.    0.    0.    0.    0.001 0.965]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "4500 iteration, USE_EMA: True, train/sup_loss: 0.0618, train/unsup_loss: 0.2132, train/total_loss: 0.2750, train/util_ratio: 0.8750, train/run_time: 0.1411, eval/loss: 0.2234, eval/top-1-acc: 0.9364, eval/balanced_acc: 0.9364, eval/precision: 0.9390, eval/recall: 0.9364, eval/F1: 0.9358, lr: 0.0002, train/prefecth_time: 0.0038 BEST_EVAL_ACC: 0.9364, at 4500 iters\n",
      "4550 iteration USE_EMA: True, train/sup_loss: 0.0178, train/unsup_loss: 0.0903, train/total_loss: 0.1081, train/util_ratio: 0.8438, train/run_time: 0.1419, lr: 0.0002, train/prefecth_time: 0.0038 \n",
      "4600 iteration USE_EMA: True, train/sup_loss: 0.0589, train/unsup_loss: 0.3741, train/total_loss: 0.4330, train/util_ratio: 0.6875, train/run_time: 0.1414, lr: 0.0002, train/prefecth_time: 0.0036 \n",
      "4650 iteration USE_EMA: True, train/sup_loss: 0.0009, train/unsup_loss: 0.0721, train/total_loss: 0.0730, train/util_ratio: 0.7500, train/run_time: 0.1405, lr: 0.0001, train/prefecth_time: 0.0035 \n",
      "4700 iteration USE_EMA: True, train/sup_loss: 0.0007, train/unsup_loss: 0.2161, train/total_loss: 0.2168, train/util_ratio: 0.8750, train/run_time: 0.1413, lr: 0.0001, train/prefecth_time: 0.0040 \n",
      "4750 iteration USE_EMA: True, train/sup_loss: 0.0015, train/unsup_loss: 0.1537, train/total_loss: 0.1552, train/util_ratio: 0.6875, train/run_time: 0.1407, lr: 0.0001, train/prefecth_time: 0.0039 \n",
      "4800 iteration USE_EMA: True, train/sup_loss: 0.0409, train/unsup_loss: 0.2510, train/total_loss: 0.2919, train/util_ratio: 0.8125, train/run_time: 0.1408, lr: 0.0001, train/prefecth_time: 0.0036 \n",
      "4850 iteration USE_EMA: True, train/sup_loss: 0.0002, train/unsup_loss: 0.2873, train/total_loss: 0.2875, train/util_ratio: 0.7812, train/run_time: 0.1418, lr: 0.0001, train/prefecth_time: 0.0039 \n",
      "4900 iteration USE_EMA: True, train/sup_loss: 0.0004, train/unsup_loss: 0.0784, train/total_loss: 0.0788, train/util_ratio: 0.7812, train/run_time: 0.1412, lr: 0.0001, train/prefecth_time: 0.0037 \n",
      "4950 iteration USE_EMA: True, train/sup_loss: 0.0006, train/unsup_loss: 0.1122, train/total_loss: 0.1128, train/util_ratio: 0.7812, train/run_time: 0.1413, lr: 0.0001, train/prefecth_time: 0.0039 \n",
      "validating...\n",
      "confusion matrix:\n",
      "[[0.957 0.002 0.002 0.001 0.    0.    0.    0.    0.022 0.016]\n",
      " [0.    0.989 0.    0.    0.    0.    0.    0.    0.001 0.01 ]\n",
      " [0.043 0.    0.783 0.007 0.132 0.01  0.019 0.005 0.    0.001]\n",
      " [0.003 0.001 0.003 0.871 0.016 0.077 0.024 0.001 0.002 0.002]\n",
      " [0.    0.    0.003 0.007 0.947 0.007 0.007 0.028 0.001 0.   ]\n",
      " [0.    0.    0.001 0.019 0.006 0.96  0.002 0.012 0.    0.   ]\n",
      " [0.002 0.    0.003 0.    0.    0.002 0.992 0.    0.001 0.   ]\n",
      " [0.005 0.    0.002 0.005 0.006 0.032 0.    0.948 0.001 0.001]\n",
      " [0.006 0.008 0.    0.    0.    0.    0.    0.    0.977 0.009]\n",
      " [0.003 0.033 0.    0.    0.    0.    0.    0.    0.001 0.963]]\n",
      "model saved: ./saved_models/fixmatch/latest_model.pth\n",
      "model saved: ./saved_models/fixmatch/model_best.pth\n",
      "5000 iteration, USE_EMA: True, train/sup_loss: 0.0005, train/unsup_loss: 0.2451, train/total_loss: 0.2457, train/util_ratio: 0.7500, train/run_time: 0.1409, eval/loss: 0.2223, eval/top-1-acc: 0.9387, eval/balanced_acc: 0.9387, eval/precision: 0.9410, eval/recall: 0.9387, eval/F1: 0.9380, lr: 0.0001, train/prefecth_time: 0.0034 BEST_EVAL_ACC: 0.9387, at 5000 iters\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2023-07-19 23:13:01,538 INFO] confusion matrix\n",
      "[2023-07-19 23:13:01,540 INFO] [[0.956 0.004 0.    0.001 0.    0.    0.    0.    0.022 0.017]\n",
      " [0.    0.99  0.    0.    0.    0.    0.    0.    0.    0.01 ]\n",
      " [0.046 0.    0.77  0.01  0.134 0.016 0.018 0.004 0.001 0.001]\n",
      " [0.003 0.002 0.002 0.872 0.015 0.079 0.023 0.    0.002 0.002]\n",
      " [0.    0.    0.002 0.007 0.949 0.007 0.008 0.026 0.001 0.   ]\n",
      " [0.    0.    0.001 0.018 0.005 0.965 0.002 0.009 0.    0.   ]\n",
      " [0.002 0.    0.003 0.001 0.    0.002 0.991 0.    0.001 0.   ]\n",
      " [0.004 0.    0.002 0.005 0.005 0.03  0.    0.951 0.001 0.002]\n",
      " [0.006 0.008 0.    0.    0.    0.    0.    0.    0.977 0.009]\n",
      " [0.003 0.034 0.    0.    0.    0.    0.    0.    0.001 0.962]]\n",
      "[2023-07-19 23:13:01,542 INFO] evaluation metric\n",
      "[2023-07-19 23:13:01,543 INFO] acc: 0.9383\n",
      "[2023-07-19 23:13:01,543 INFO] precision: 0.9409\n",
      "[2023-07-19 23:13:01,544 INFO] recall: 0.9383\n",
      "[2023-07-19 23:13:01,544 INFO] f1: 0.9375\n",
      "[2023-07-19 23:13:02,617 INFO] Best acc 0.9387 at epoch 0\n",
      "[2023-07-19 23:13:02,618 INFO] Training finished.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model saved: ./saved_models/fixmatch/latest_model.pth\n"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(config, algorithm)\n",
    "trainer.fit(train_lb_loader, train_ulb_loader, eval_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Step 5: evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2023-07-19 23:14:07,511 INFO] confusion matrix\n",
      "[2023-07-19 23:14:07,513 INFO] [[0.956 0.004 0.    0.001 0.    0.    0.    0.    0.022 0.017]\n",
      " [0.    0.99  0.    0.    0.    0.    0.    0.    0.    0.01 ]\n",
      " [0.046 0.    0.77  0.01  0.134 0.016 0.018 0.004 0.001 0.001]\n",
      " [0.003 0.002 0.002 0.872 0.015 0.079 0.023 0.    0.002 0.002]\n",
      " [0.    0.    0.002 0.007 0.949 0.007 0.008 0.026 0.001 0.   ]\n",
      " [0.    0.    0.001 0.018 0.005 0.965 0.002 0.009 0.    0.   ]\n",
      " [0.002 0.    0.003 0.001 0.    0.002 0.991 0.    0.001 0.   ]\n",
      " [0.004 0.    0.002 0.005 0.005 0.03  0.    0.951 0.001 0.002]\n",
      " [0.006 0.008 0.    0.    0.    0.    0.    0.    0.977 0.009]\n",
      " [0.003 0.034 0.    0.    0.    0.    0.    0.    0.001 0.962]]\n",
      "[2023-07-19 23:14:07,515 INFO] evaluation metric\n",
      "[2023-07-19 23:14:07,516 INFO] acc: 0.9383\n",
      "[2023-07-19 23:14:07,517 INFO] precision: 0.9409\n",
      "[2023-07-19 23:14:07,517 INFO] recall: 0.9383\n",
      "[2023-07-19 23:14:07,518 INFO] f1: 0.9375\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'acc': 0.9383,\n",
       " 'precision': 0.9408766240300274,\n",
       " 'recall': 0.9382999999999999,\n",
       " 'f1': 0.9375322532701483}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.evaluate(eval_loader)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Step 6: predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "y_pred, y_logits = trainer.predict(eval_loader)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "test",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  },
  "vscode": {
   "interpreter": {
    "hash": "efd87a861e5021e4a438e5b61d692cea261dd91508182bfdfdb13fb969975ffe"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
